/**
 * 
 */
package com.ramon.heritrix;

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.net.MalformedURLException;
import java.security.NoSuchAlgorithmException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Random;

import com.ramon.ohcda.util.Constants;
import com.ramon.ohcda.util.DataHandler;

/**
 * @author Ramon
 */
public class ParseFiles {

    private File dir;

    private static String filenameExpansion = "arc.gz";

    private List<File> arcFileList;

    private List<String> urlList;

    private HashMap hm;

    private Date now;

    private SimpleDateFormat formatter;

    private SimpleDateFormat filenameformatter;

    private static boolean debug = false;

    private static int counter = 0;

    public ParseFiles(String directory) {
        dir = new File(directory);
        arcFileList = new LinkedList();
        hm = new HashMap();
        urlList = new LinkedList<String>();
        formatter = new SimpleDateFormat("HH:mm:ss");
        filenameformatter = new SimpleDateFormat("yyyyMMddHHmmss");
        print("System initializing...");
    }

    public void setDebug(boolean debug) {
        this.debug = debug;
    }

    public void getFilesReady() {
        print("Collecting files for analysing...");
        for (File f : dir.listFiles()) {
            if (f.getName().endsWith(filenameExpansion)) {
                arcFileList.add(f);
                print(f.getName() + " ready...");
            }
        }
    }

    public void analyse() throws MalformedURLException, IOException,
            NoSuchAlgorithmException {
        for (File f : arcFileList) {
            counter++;
            print("parsing " + f.getName() + " ...[" + counter + "/"
                    + arcFileList.size() + "]");
            ArcParser ap = new ArcParser(f);
            ap.parse();
            List<String> tmpUrlList = ap.getUrlList();
            for (String s : tmpUrlList) {
                urlList.add(s);
            }
            HashMap tmpHM = ap.getHashMap();
            for (Object url : tmpHM.keySet().toArray()) {
                hm.put(url, tmpHM.get(url));
            }
            ap.destroy();
            print(tmpUrlList.size() + " URLs get fetched...");

        }
    }

    private void generateUrlListsAsFile() throws IOException {
        String filename = "";
        filename = filenameformatter.format(now);
        filename = "urls-" + filename + ".txt";
        FileWriter fw = new FileWriter(new File("urls/" + filename));
        for (String url : urlList) {
            fw.write(url + "\r\n");
        }
        fw.close();
    }

    private void generateDataSet() {
        try {
            DataHandler.write(hm);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    private void generateUrlListsAsFileWithChangeFlag() throws IOException {
        String filename = "";
        filename = filenameformatter.format(now);
        filename = "urlswithchangeflag-" + filename + ".txt";
        FileWriter fw = new FileWriter(new File("urls/" + filename));
        Random r = new Random();
        for (String url : urlList) {
            if (r.nextBoolean()) {
                fw.write(url + " 1\r\n");
            } else {
                fw.write(url + " 0\r\n");
            }

        }
        fw.close();
    }

    private void showStatistics() {
        print("We've susccessfully crawled " + urlList.size() + " urls");
    }

    private void print(String msg) {
        if (debug) {
            now = Calendar.getInstance().getTime();
            System.out.println("[" + formatter.format(now) + "] " + msg);
        }
    }

    public static void main(String[] args) throws MalformedURLException,
            IOException, NoSuchAlgorithmException {
        String filename = null;
        if (System.getProperty("os.name").equals("Windows XP")) {
            filename = Constants.windowsArcPath;
        } else if (System.getProperty("os.name").equals("Mac OS X")) {
            filename = Constants.macArcPath;
        }
        ParseFiles pf = new ParseFiles(filename);
        pf.setDebug(true);
        pf.getFilesReady();
        pf.analyse();
        pf.showStatistics();
        pf.generateUrlListsAsFile();
        pf.generateDataSet();
    }
}
